import os
import numpy as np
import pandas as pd
import seaborn; seaborn.set_theme()
import plotly.io; plotly.io.templates.default = "seaborn"
import emat
import yaml
from emat.util.show_dir import show_dir
from emat.analysis import display_experiments
import logging
from emat.util.loggers import log_to_stderr
log = log_to_stderr(logging.INFO)
import emat_vestate
import asyncio
database_path = os.path.expanduser("~/EMAT-VE/ve2-rspm-2022-02-23.db")
initialize = not os.path.exists(database_path)
db = emat.SQLiteDB(database_path, initialize=initialize)
model_scope = emat.Scope(os.path.join(os.getcwd(),'vestate-emat-files', 'odot-otp-scope.yml'))
fx = emat_vestate.VEStateModel(db=db, scope=model_scope)
[00:29.12] MainProcess/INFO: running script emat_db_init.sql [00:29.13] MainProcess/INFO: running script meta_model.sql [00:29.14] MainProcess/INFO: found no experiments with missing run_id's [00:29.14] MainProcess/INFO: running script emat_db_init_views.sql [00:29.19] MainProcess/WARNING: changing cwd to E:\Projects\Clients\odot\ODOT_VEState\Github\sources\models\VE-State-EMAT\Temporary\tmpug0jszi7
model_scope.info()
name: VEState desc: VisionEval State Strategic Planning Model uncertainties: LUDENSITYMIX = 0.0 to 0.3 INTDENSITYSCEN = 0.8 to 1.0 HHPOPGROWTHRATE = categorical INCOMEGROWTHRATE = 0.9 to 1.15 LDVECODRVSCEN = 0.0 to 1.0 measures: HouseholdDvmt HouseholdCarSvcDvmt ComSvcDvmt PTVanDvmt LdvDvmt HvyTruckDvmt BusDvmt TotalDvmt TotalGGE AverageLdvMpg HouseholdCO2e MetroHhDvmt MetroAveHhDvmtPerHh MetroAveHhDvmtPerPrsn MetroAveHhDvmtPerVeh MetroHhGGE MetroHhCO2e
# design1 = fx.design_experiments(n_samples=300, design_name='exp_300')
# design1 = fx.design_experiments(n_samples=30, design_name='exp_30')
design1 = fx.db.read_experiment_all(fx.scope.name,design_name='exp_300')
design1
| LUDENSITYMIX | INTDENSITYSCEN | HHPOPGROWTHRATE | INCOMEGROWTHRATE | LDVECODRVSCEN | |
|---|---|---|---|---|---|
| experiment | |||||
| 1 | 0.256121 | 0.990824 | mid | 0.919153 | 0.736541 |
| 2 | 0.189457 | 0.943774 | high | 1.127183 | 0.328131 |
| 3 | 0.044362 | 0.809831 | low | 0.951936 | 0.520699 |
| 4 | 0.178956 | 0.893921 | mid | 0.923353 | 0.355188 |
| 5 | 0.032498 | 0.860648 | mid | 0.953031 | 0.618596 |
| ... | ... | ... | ... | ... | ... |
| 296 | 0.228483 | 0.952907 | mid | 1.099318 | 0.834123 |
| 297 | 0.086440 | 0.898342 | high | 0.970182 | 0.965528 |
| 298 | 0.267638 | 0.886844 | low | 1.049866 | 0.590587 |
| 299 | 0.039768 | 0.938527 | high | 1.125158 | 0.330551 |
| 300 | 0.152111 | 0.866245 | high | 1.079549 | 0.589367 |
300 rows × 5 columns
# params = design1.loc[1,['LUDENSITYMIX', 'INTDENSITYSCEN', 'HHPOPGROWTHRATE', 'INCOMEGROWTHRATE']].to_dict()
# # params = {key:params.get(key) for key in ['LUDENSITYMIX', 'INTDENSITYSCEN', 'HHPOPGROWTHRATE', 'INCOMEGROWTHRATE']}
# params
# fx.setup(params)
# fx.run()
# fx.load_measures()
background = fx.async_experiments(
design=design1,
max_n_workers=15,
batch_size=1,
)
[00:49.09] MainProcess/INFO: asynchronous_experiments(max_n_workers=15) [00:49.10] MainProcess/INFO: AsyncExperimentalDesign.run start [00:49.10] MainProcess/INFO: initializing default DistributedEvaluator.client [00:49.10] MainProcess/INFO: max_n_workers=15, actual n_workers=15 [00:49.10] MainProcess/INFO: n_workers=15 C:\Users\aditya.gore\.conda\envs\emat\lib\site-packages\distributed\node.py:160: UserWarning: Port 8787 is already in use. Perhaps you already have a cluster running? Hosting the HTTP server on port 60489 instead [00:50.96] MainProcess/INFO: completed initializing default DistributedEvaluator.client [00:54.27] MainProcess/INFO: AsyncExperimentalDesign.run dispatching experiments [00:54.36] MainProcess/INFO: performing 300 scenarios/policies * 1 model(s) = 300 experiments [00:54.37] MainProcess/INFO: experiments in asynchronous evaluator [00:54.38] MainProcess/INFO: AsyncExperimentalDesign.run dispatching task complete
background.progress() # Initially everything is pending
'300 runs: 130 done, 170 failed'
background.status()
experiment
1 done
2 FAILED EXPERIMENT 2: Command '['Rscript.exe', ...
3 FAILED EXPERIMENT 3: Command '['Rscript.exe', ...
4 done
5 FAILED EXPERIMENT 5: Command '['Rscript.exe', ...
...
296 FAILED EXPERIMENT 296: Command '['Rscript.exe'...
297 FAILED EXPERIMENT 297: Command '['Rscript.exe'...
298 FAILED EXPERIMENT 298: Command '['Rscript.exe'...
299 FAILED EXPERIMENT 299: Command '['Rscript.exe'...
300 FAILED EXPERIMENT 300: Command '['Rscript.exe'...
Length: 300, dtype: object
results = await background.final_results()
results
C:\Users\aditya.gore\.conda\envs\emat\lib\site-packages\emat\model\asynchronous.py:157: UserWarning: 300 runs: 130 done, 170 failed
| LUDENSITYMIX | INTDENSITYSCEN | HHPOPGROWTHRATE | INCOMEGROWTHRATE | LDVECODRVSCEN | HouseholdDvmt | HouseholdCarSvcDvmt | ComSvcDvmt | PTVanDvmt | LdvDvmt | ... | TotalDvmt | TotalGGE | AverageLdvMpg | HouseholdCO2e | MetroHhDvmt | MetroAveHhDvmtPerHh | MetroAveHhDvmtPerPrsn | MetroAveHhDvmtPerVeh | MetroHhGGE | MetroHhCO2e | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| experiment | |||||||||||||||||||||
| 1 | 0.256121 | 0.990824 | mid | 0.919153 | 0.736541 | 5.485794e+07 | 452165.589148 | 5.923499e+06 | 87334.962793 | 6.086877e+07 | ... | 6.335914e+07 | 1.277835e+06 | 48.780445 | 1.313707e+10 | 1.852563e+07 | 24.510340 | 9.758614 | 12.317947 | 403111.340918 | 1.785584e+06 |
| 2 | 0.189457 | 0.943774 | high | 1.127183 | 0.328131 | NaN | NaN | NaN | NaN | NaN | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 3 | 0.044362 | 0.809831 | low | 0.951936 | 0.520699 | NaN | NaN | NaN | NaN | NaN | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 4 | 0.178956 | 0.893921 | mid | 0.923353 | 0.355188 | 5.694313e+07 | 477227.710243 | 5.923499e+06 | 87334.962793 | 6.295396e+07 | ... | 6.560023e+07 | 1.320991e+06 | 48.764963 | 1.364755e+10 | 1.930274e+07 | 25.538497 | 10.169451 | 12.767357 | 420617.804350 | 1.861412e+06 |
| 5 | 0.032498 | 0.860648 | mid | 0.953031 | 0.618596 | NaN | NaN | NaN | NaN | NaN | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 296 | 0.228483 | 0.952907 | mid | 1.099318 | 0.834123 | NaN | NaN | NaN | NaN | NaN | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 297 | 0.086440 | 0.898342 | high | 0.970182 | 0.965528 | NaN | NaN | NaN | NaN | NaN | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 298 | 0.267638 | 0.886844 | low | 1.049866 | 0.590587 | NaN | NaN | NaN | NaN | NaN | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 299 | 0.039768 | 0.938527 | high | 1.125158 | 0.330551 | NaN | NaN | NaN | NaN | NaN | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 300 | 0.152111 | 0.866245 | high | 1.079549 | 0.589367 | NaN | NaN | NaN | NaN | NaN | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
300 rows × 22 columns
from emat.model import create_metamodel
from emat.util.loggers import timing_log
with timing_log("create metamodel"):
mm = create_metamodel(fx.scope, results[~results.HouseholdDvmt.isna()], suppress_converge_warnings=True)
[09:58:55.30] MainProcess/CRITICAL: <TIME BEGINS> create metamodel [09:58:55.30] MainProcess/INFO: creating metamodel from data [10:00:27.31] MainProcess/CRITICAL: < TIME ENDS > create metamodel <1m 32.01s>
with timing_log("crossvalidate metamodel"):
display(mm.cross_val_scores())
[10:00:32.11] MainProcess/CRITICAL: <TIME BEGINS> crossvalidate metamodel
| Cross Validation Score | |
|---|---|
| HouseholdDvmt | 1.0000 |
| HouseholdCarSvcDvmt | 0.9996 |
| ComSvcDvmt | 1.0000 |
| PTVanDvmt | 1.0000 |
| LdvDvmt | 1.0000 |
| HvyTruckDvmt | 0.9998 |
| BusDvmt | 1.0000 |
| TotalDvmt | 1.0000 |
| TotalGGE | 1.0000 |
| AverageLdvMpg | 0.9851 |
| HouseholdCO2e | 1.0000 |
| MetroHhDvmt | 1.0000 |
| MetroAveHhDvmtPerHh | 1.0000 |
| MetroAveHhDvmtPerPrsn | 1.0000 |
| MetroAveHhDvmtPerVeh | 1.0000 |
| MetroHhGGE | 0.9999 |
| MetroHhCO2e | 0.9999 |
[10:01:44.13] MainProcess/CRITICAL: < TIME ENDS > crossvalidate metamodel <1m 12.02s>
from emat.analysis import display_experiments
display_experiments(fx.scope, results)
from emat.workbench.analysis import cart
of_interest = results.HouseholdDvmt.isna()
cart_alg = cart.CART(
fx.read_experiment_parameters(design_name='exp_300'),
of_interest,
)
cart_alg.build_tree()
from emat.util.xmle import Show
Show(cart_alg.show_tree(format='svg'))
cart_alg.boxes_to_dataframe(include_stats=True)
| Box Statistics | INCOMEGROWTHRATE | LDVECODRVSCEN | ||||||||
|---|---|---|---|---|---|---|---|---|---|---|
| coverage | density | gini | entropy | res dim | mass | min | max | min | max | |
| box 0 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 1 | 0.166667 | NaN | 0.941627 | NaN | NaN |
| box 1 | 0.111765 | 1.000000 | 0.000000 | 0.000000 | 2 | 0.063333 | 0.941627 | 0.976491 | NaN | 0.55213 |
| box 2 | 0.117647 | 0.869565 | 0.226843 | 0.558629 | 2 | 0.076667 | 0.941627 | 0.976491 | 0.55213 | NaN |
| box 3 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 1 | 0.256667 | 0.976491 | 1.04097 | NaN | NaN |
| box 4 | 0.770588 | 1.000000 | 0.000000 | 0.000000 | 1 | 0.436667 | 1.04097 | NaN | NaN | NaN |
valid_exp = ~results.HouseholdDvmt.isna()
of_interest = (results[valid_exp].HouseholdCO2e < 30E9)
cart_alg = cart.CART(
fx.read_experiment_parameters(design_name='exp_300')[valid_exp],
of_interest,
)
cart_alg.build_tree()
Show(cart_alg.show_tree(format='svg'))
cart_alg.boxes_to_dataframe(include_stats=True)
| Box Statistics | HHPOPGROWTHRATE | INCOMEGROWTHRATE | ||||||||
|---|---|---|---|---|---|---|---|---|---|---|
| coverage | density | gini | entropy | res dim | mass | min | max | min | max | |
| box 0 | 0.509091 | 1.00000 | 0.000000 | 0.000000 | 2 | 0.430769 | {mid, low} | {mid, high, low} | NaN | 1.001809 |
| box 1 | 0.200000 | 1.00000 | 0.000000 | 0.000000 | 2 | 0.169231 | {mid, low} | {mid, low} | 1.001809 | 1.029172 |
| box 2 | 0.045455 | 0.50000 | 0.500000 | 1.000000 | 2 | 0.076923 | {mid, low} | {mid, low} | 1.029172 | NaN |
| box 3 | 0.245455 | 0.84375 | 0.263672 | 0.625262 | 2 | 0.246154 | {mid, low} | {mid, high, low} | 1.001809 | NaN |